package crawler;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ResultItemsCollectorPipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.JsonPathSelector;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.HashSet;
import java.util.List;
import java.util.Set;


public class BilibiliProcessor implements PageProcessor {
	public static Set<String> set=new HashSet();
    private static Site site = Site.me().setRetryTimes(3).setSleepTime(1000);

    public Site getSite() {
        return site;
    }

    public void process(Page page) {
        String a = page.getJson().toString();
        String s = page.getUrl().get();
        String substring = s.substring(s.lastIndexOf("=") + 1);
//        a = a.replace("jQuery17207315492635561198_1509009462091(", "").replace(")", "");
        a = a.replace(a.substring(0,a.indexOf("(")+1), "").replace(")", "");
        //获取的如果是列表页
        if (page.getUrl().regex("https://api.bilibili.com/archive_rank/").match()) {
            //对列表页，进行分页
            if (substring.equals("1"))
                for (int i = 1; i < 4; i++) {
                	Request request = new Request();
                    request.setExtras(page.getRequest().getExtras());
                    request.setUrl("https://api.bilibili.com/archive_rank/getarchiverankbypartion?callback=jQuery17207315492635561198_1509009462091&type=jsonp&tid=138&_=1509009463756&pn=" + i);
//                  https://api.bilibili.com/archive_rank/getarchiverankbypartion?tid=138&pn=1
                    page.addTargetRequest(request);
                }
            //将列表页获取的数据进行解析
            List<String> list = new JsonPathSelector("$.data.archives[*].aid").selectList(a);
            //将视频信息放入队列
            for (String url : list) {
                Request request = page.getRequest();
                request.setUrl("https://api.bilibili.com/playurl?callback=callbackfunction&aid=" + url + "&page=1&platform=html5&quality=1&vtype=mp4&type=json&token=5ec90a52daa6eb3de19ea732c518c741")
                		.addHeader("Referer","https://m.bilibili.com/video/av"+url+".html");
                page.addTargetRequest(request);
//                System.out.println(page);
              //  System.out.println("真正："+request.getUrl());
                System.out.println("遍历:"+"https://api.bilibili.com/playurl?callback=callbackfunction&aid=" + url + "&page=1&platform=html5&quality=1&vtype=mp4&type=json&token=5ec90a52daa6eb3de19ea732c518c741");
                	String s1="https://api.bilibili.com/playurl?callback=callbackfunction&aid=" + url + "&page=1&platform=html5&quality=1&vtype=mp4&type=json&token=5ec90a52daa6eb3de19ea732c518c741";
                	//f1(s1);
                	set.add(s1);
             //   List<String> urls = new JsonPathSelector("$.durl[*].url").selectList(a);
                //System.out.println(urls);
            }
        } else {
            
        	System.out.println("Sorry,你走到了一条死胡同！！");
        	

        }
    }
    public static void  f1(Set<String> set) throws Exception {
    	FileOutputStream fos=new FileOutputStream(new File("D:/bzhan.txt"));
        OutputStreamWriter osw=new OutputStreamWriter(fos, "UTF-8");
        BufferedWriter  bw=new BufferedWriter(osw);
        
        //简写如下：
        //BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(
        //        new FileOutputStream(new File("E:/phsftp/evdokey/evdokey_201103221556.txt")), "UTF-8"));

       for(String string : set){
            bw.write(string.toString().trim()+"\t\n");
       }
        
        //注意关闭的先后顺序，先打开的后关闭，后打开的先关闭
        bw.close();
        osw.close();
        fos.close();
    }

	public static void main(String[] args) throws Exception {
        Request request = new Request();
        request.setUrl("https://api.bilibili.com/archive_rank/getarchiverankbypartion?callback=jQuery17207315492635561198_1509009462091&type=jsonp&tid=138&_=1509009463756&pn=1")
//        https://api.bilibili.com/archive_rank/getarchiverankbypartion?tid=138&pn=1
                .addHeader("Referer", "https://www.bilibili.com/video/ent_funny_1.html")
                .addHeader("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) FxiOS/40.0 Mobile/12F69 Safari/600.1.4");
        		
        //运行爬虫
        Spider.create(new BilibiliProcessor())
                .addRequest(request)
                .addPipeline(new ResultItemsCollectorPipeline())
                .thread(5)
                .run();
        	f1(set);
    }
}










 




